<!DOCTYPE html>
<html lang="en">
  <head>
  <meta charset="utf-8" />
  <meta http-equiv="X-UA-Compatible" content="IE=edge">
  <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0">


  <title>Kubernetes使用Prometheus搭建监控平台-blog.qikqiak.com|阳明的博客|Kubernetes|Docker|Python|Golang|Cloud Native</title>
  <meta property="og:title" content="Kubernetes使用Prometheus搭建监控平台" />
  <meta name="twitter:title" content="Kubernetes使用Prometheus搭建监控平台" />

  <meta name="description" content="最近在测试环境搭建了Kubernetes集群环境，迁移了部分测试环境的应用，由于测试集群性能不是很好，有时会遇到集群资源不够的情况，一般情况下我们是直接通过Dashboard的资源统计图标进行观察的，但是很显然如果要上到生产环境，就需要更自动化的方式来对集群、Pod甚至容器进行监控了。Kubernetes内置了一套监控方案：influxdb&#43;grafana&#43;heapster。但由于之前我们的应用的业务监控使用的是Prometheus，所以这里准备使用Prometheus来完成k8s的集群监控。">
  <meta property="og:description" content="最近在测试环境搭建了Kubernetes集群环境，迁移了部分测试环境的应用，由于测试集群性能不是很好，有时会遇到集群资源不够的情况，一般情况下我们是直接通过Dashboard的资源统计图标进行观察的，但是很显然如果要上到生产环境，就需要更自动化的方式来对集群、Pod甚至容器进行监控了。Kubernetes内置了一套监控方案：influxdb&#43;grafana&#43;heapster。但由于之前我们的应用的业务监控使用的是Prometheus，所以这里准备使用Prometheus来完成k8s的集群监控。">
  <meta name="twitter:description" content="最近在测试环境搭建了Kubernetes集群环境，迁移了部分测试环境的应用，由于测试集群性能不是很好，有时会遇到集群资源不够的情况，一般情况下我们是直接通过Dashboard的资源统计图标进行观察的，但是很显然如果要上到生产环境，就需要更自动化的方式来对集群、Pod甚至容器进行监控了。Kubernetes内置了一套监控方案：influxdb&#43;grafana&#43;heapster。但由于之前我们的应用的 …">
  <meta name="author" content=""/>
  <link href='https://blog.qikqiak.com/img/favicon.ico' rel='icon' type='image/x-icon'/>
  <meta property="og:image" content="https://blog.qikqiak.com/img/avatar.jpeg" />
  <meta name="twitter:image" content="https://blog.qikqiak.com/img/avatar.jpeg" />
  <meta name="twitter:card" content="summary" />
  <meta property="og:url" content="https://blog.qikqiak.com/post/kubernetes-monitor-prometheus-grafana/" />
  <meta property="og:type" content="website" />
  <meta property="og:site_name" content="阳明的博客" />

  <meta name="generator" content="Hugo 0.34" />
  <link rel="canonical" href="https://blog.qikqiak.com/post/kubernetes-monitor-prometheus-grafana/" />
  <link rel="alternate" href="https://blog.qikqiak.com/index.xml" type="application/rss+xml" title="阳明的博客">
  <link href="https://cdn.bootcss.com/KaTeX/0.7.1/katex.min.css" rel="stylesheet">
  <link href="https://cdn.bootcss.com/font-awesome/4.7.0/css/font-awesome.min.css" rel="stylesheet">
  <link href="https://cdn.bootcss.com/bootstrap/3.3.7/css/bootstrap.min.css" rel="stylesheet">
  <link rel="stylesheet" href="https://blog.qikqiak.com/css/main.css?t=1235" />
  <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Lora:400,700,400italic,700italic" />
  <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Open+Sans:300italic,400italic,600italic,700italic,800italic,400,300,600,700,800" />
  <link rel="stylesheet" href="https://blog.qikqiak.com/css/pygment_highlights.css" />
  <link rel="stylesheet" href="https://blog.qikqiak.com/css/highlight.min.css" />
  <link rel="stylesheet" href="https://blog.qikqiak.com/css/prism.css?t=123" />
  <link rel="stylesheet" href="https://blog.qikqiak.com/css/search.css" />
<meta name="keywords" content="kubernetes, 监控, prometheus, grafana, influxdb">
<script>
var _hmt = _hmt || [];
(function() {
  var hm = document.createElement("script");
  hm.src = "https://hm.baidu.com/hm.js?970de9920f6089c287a7fd366138d4e9";
  var s = document.getElementsByTagName("script")[0];
  s.parentNode.insertBefore(hm, s);
})();
</script>



<link href="https://cdn.bootcss.com/photoswipe/4.1.1/photoswipe.min.css" rel="stylesheet">
<link href="https://cdn.bootcss.com/photoswipe/4.1.1/default-skin/default-skin.min.css" rel="stylesheet">



<div class="pswp" tabindex="-1" role="dialog" aria-hidden="true">

<div class="pswp__bg"></div>

<div class="pswp__scroll-wrap">
    
    <div class="pswp__container">
      <div class="pswp__item"></div>
      <div class="pswp__item"></div>
      <div class="pswp__item"></div>
    </div>
    
    <div class="pswp__ui pswp__ui--hidden">
    <div class="pswp__top-bar">
      
      <div class="pswp__counter"></div>
      <button class="pswp__button pswp__button--close" title="Close (Esc)"></button>
      <button class="pswp__button pswp__button--share" title="Share"></button>
      <button class="pswp__button pswp__button--fs" title="Toggle fullscreen"></button>
      <button class="pswp__button pswp__button--zoom" title="Zoom in/out"></button>
      
      
      <div class="pswp__preloader">
        <div class="pswp__preloader__icn">
          <div class="pswp__preloader__cut">
            <div class="pswp__preloader__donut"></div>
          </div>
        </div>
      </div>
    </div>
    <div class="pswp__share-modal pswp__share-modal--hidden pswp__single-tap">
      <div class="pswp__share-tooltip"></div>
    </div>
    <button class="pswp__button pswp__button--arrow--left" title="Previous (arrow left)">
    </button>
    <button class="pswp__button pswp__button--arrow--right" title="Next (arrow right)">
    </button>
    <div class="pswp__caption">
      <div class="pswp__caption__center"></div>
    </div>
    </div>
    </div>
</div>

<script>
window.ga=window.ga||function(){(ga.q=ga.q||[]).push(arguments)};ga.l=+new Date;
ga('create', 'UA-69668147-3', 'auto');
ga('send', 'pageview');
</script>
<script async src='//www.google-analytics.com/analytics.js'></script>

</head>

  <body>
    <nav class="navbar navbar-default navbar-fixed-top navbar-custom">
  <div class="container-fluid">
    <div class="navbar-header">
      <button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#main-navbar">
        <span class="sr-only">Toggle navigation</span>
        <span class="icon-bar"></span>
        <span class="icon-bar"></span>
        <span class="icon-bar"></span>
      </button>
      <a class="navbar-brand" href="https://blog.qikqiak.com/">阳明的博客</a>
    </div>

    <div class="collapse navbar-collapse" id="main-navbar">
      <ul class="nav navbar-nav navbar-right">
        
          
            <li>
              <a title="Home" href="https://ydzs.io">Home</a>
            </li>
          
        
          
            <li class="navlinks-container">
              <a class="navlinks-parent" href="javascript:void(0)">Categories</a>
              <div class="navlinks-children">
                
                  <a href="https://blog.qikqiak.com/page/archive/">Archive</a>
                
                  <a href="https://blog.qikqiak.com/tags">tags</a>
                
                  <a href="https://blog.qikqiak.com/tags/kubernetes">kubernetes</a>
                
                  <a href="https://blog.qikqiak.com/tags/python">python</a>
                
                  <a href="https://blog.qikqiak.com/tags/django">django</a>
                
                  <a href="https://blog.qikqiak.com/tags/ops">devops</a>
                
              </div>
            </li>
          
        
          
            <li class="navlinks-container">
              <a class="navlinks-parent" href="javascript:void(0)">海马学院</a>
              <div class="navlinks-children">
                
                  <a href="https://www.haimaxy.com/course/6y02wp/?utm_source=blog&amp;utm_campaign=referral&amp;utm_medium=topmenu">Python微服务实战</a>
                
                  <a href="https://www.haimaxy.com/course/pww13p/?utm_source=blog&amp;utm_campaign=referral&amp;utm_medium=topmenu">Webpack3.x 入门</a>
                
                  <a href="https://www.haimaxy.com/course/pjrqxm/?utm_source=blog&amp;utm_campaign=referral&amp;utm_medium=topmenu">Kubernetes 集群搭建</a>
                
              </div>
            </li>
          
        
          
            <li>
              <a title="About" href="https://blog.qikqiak.com/page/about/">About</a>
            </li>
          
        

        

        

        
          <li>
            <a href="#modalSearch" data-toggle="modal" data-target="#modalSearch" style="outline: none;">
              <span class="hidden-sm hidden-md hidden-lg">Search</span> <span id="searchGlyph" class="glyphicon glyphicon-search"></span>
            </a>
          </li>
          

      </ul>
    </div>

    <div class="avatar-container">
      <div class="avatar-img-border">
        
          <a title="阳明的博客" href="https://blog.qikqiak.com/">
            <img class="avatar-img" src="https://blog.qikqiak.com/img/avatar.jpeg" alt="阳明的博客" />
          </a>
        
      </div>
    </div>

  </div>
</nav>





  <div id="modalSearch" class="modal fade" role="dialog">
    <div class="modal-dialog">
      <div class="modal-content">
        <div class="modal-header">
          <button type="button" class="close" data-dismiss="modal">&times;</button>
          <h4 class="modal-title">Search blog.qikqiak.com</h4>
        </div>
        <div class="modal-body">
            
<div class="aa-input-container" id="aa-input-container">
    <input type="search" id="aa-search-input" class="aa-input-search" placeholder="Search for titles or URIs..." name="search" autocomplete="off" />
    <svg class="aa-input-icon" viewBox="654 -372 1664 1664">
        <path d="M1806,332c0-123.3-43.8-228.8-131.5-316.5C1586.8-72.2,1481.3-116,1358-116s-228.8,43.8-316.5,131.5  C953.8,103.2,910,208.7,910,332s43.8,228.8,131.5,316.5C1129.2,736.2,1234.7,780,1358,780s228.8-43.8,316.5-131.5  C1762.2,560.8,1806,455.3,1806,332z M2318,1164c0,34.7-12.7,64.7-38,90s-55.3,38-90,38c-36,0-66-12.7-90-38l-343-342  c-119.3,82.7-252.3,124-399,124c-95.3,0-186.5-18.5-273.5-55.5s-162-87-225-150s-113-138-150-225S654,427.3,654,332  s18.5-186.5,55.5-273.5s87-162,150-225s138-113,225-150S1262.7-372,1358-372s186.5,18.5,273.5,55.5s162,87,225,150s113,138,150,225  S2062,236.7,2062,332c0,146.7-41.3,279.7-124,399l343,343C2305.7,1098.7,2318,1128.7,2318,1164z" />
    </svg>
</div>
<script src="https://res.cloudinary.com/jimmysong/raw/upload/rootsongjc-hugo/algoliasearch.min.js"></script>
<script src="https://res.cloudinary.com/jimmysong/raw/upload/rootsongjc-hugo/autocomplete.min.js"></script>

<script>
var client = algoliasearch("1JDRAS0AZR", "8804ac109158bb3bb60d74ce98fa332f");
var index = client.initIndex('prod_blog');

autocomplete('#aa-search-input',
{ hint: false}, {
    source: autocomplete.sources.hits(index, {hitsPerPage: 5}),
    
    displayKey: 'name',
    
    templates: {
        
        suggestion: function(suggestion) {
            return '<span>' + '<a href="https://blog.qikqiak.com/post/' + suggestion.slug + '">' +
            suggestion._highlightResult.title.value + '</a></span>';
        }
    }
});
</script>
        </div>
        <div class="modal-footer">
          <button type="button" class="btn btn-default" data-dismiss="modal">close</button>
        </div>
      </div>
    </div>
  </div>

    
  
  
  




  
    <div id="header-big-imgs" data-num-img=1 data-img-src-1="https://blog.qikqiak.com/img/posts/k8s-prometheus-grafana-cover.png" ></div>
  

  <header class="header-section has-img">
    
      <div class="intro-header big-img">
        
        <div class="container">
          <div class="row">
            <div class="col-lg-8 col-lg-offset-2 col-md-10 col-md-offset-1">
              <div class="post-heading">
                <h1>Kubernetes使用Prometheus搭建监控平台</h1>
                  
                  
                    <span class="post-meta">
  Posted on October 17, 2017
  
</span>


                  
              </div>
            </div>
          </div>
        </div>
        <span class="img-desc" style="display: inline;"></span>
      </div>
    
    <div class="intro-header no-img">
      
      <div class="container">
        <div class="row">
          <div class="col-lg-8 col-lg-offset-2 col-md-10 col-md-offset-1">
            <div class="post-heading">
              <h1>Kubernetes使用Prometheus搭建监控平台</h1>
                
                
                  <span class="post-meta">
  Posted on October 17, 2017
  
</span>


                
            </div>
          </div>
        </div>
      </div>
    </div>
  </header>


    
<div class="container" role="main">
  <div class="row">
    <div class="col-lg-8 col-lg-offset-2 col-md-10 col-md-offset-1">

      <div>
          
          
          <h5 id="tags" style="margin-top: 0px;">标签:
            
                <a href="https://blog.qikqiak.com/tags/kubernetes/">kubernetes</a> &nbsp;
            
                <a href="https://blog.qikqiak.com/tags/prometheus/">prometheus</a> &nbsp;
            
                <a href="https://blog.qikqiak.com/tags/grafana/">grafana</a> &nbsp;
            
                <a href="https://blog.qikqiak.com/tags/influxdb/">influxdb</a> &nbsp;
            
          </h5>
          
      </div>

      <article role="main" class="blog-post">
        <p>最近在测试环境搭建了<code>Kubernetes</code>集群环境，迁移了部分测试环境的应用，由于测试集群性能不是很好，有时会遇到集群资源不够的情况，一般情况下我们是直接通过Dashboard的资源统计图标进行观察的，但是很显然如果要上到生产环境，就需要更自动化的方式来对集群、Pod甚至容器进行监控了。<code>Kubernetes</code>内置了一套监控方案：influxdb+grafana+heapster。但由于之前我们的应用的业务监控使用的是<code>Prometheus</code>，所以这里准备使用<code>Prometheus</code>来完成k8s的集群监控。</p>

<p>
欢迎大家加入我们的知识星球：<code>Kubernetes</code>。
<img src="https://blog.qikqiak.com/img/xq.png" alt="知识星球" /></p>

<p>扫描下面的二维码(或微信搜索<code>k8s技术圈</code>)关注我们的微信公众帐号，在微信公众帐号中回复 <strong>加群</strong> 即可加入到我们的 kubernetes 讨论群里面共同学习。
<img src="https://blog.qikqiak.com/img/posts/qrcode_for_gh_d6dd87b6ceb4_430.jpg" alt="qrcode" /></p>

<h2 id="prometheus-简介">Prometheus 简介</h2>

<p><code>Prometheus</code>是SoundCloud开源的一款开源软件。它的实现参考了Google内部的监控实现，与源自Google的Kubernetes结合起来非常合适。另外相比influxdb的方案，性能更加突出，而且还内置了报警功能。它针对大规模的集群环境设计了拉取式的数据采集方式，你只需要在你的应用里面实现一个<code>metrics</code>接口，然后把这个接口告诉<code>Prometheus</code>就可以完成数据采集了。</p>

<h2 id="安装prometheus">安装Prometheus</h2>

<p>首先我们使用<code>ConfigMap</code>的形式来设置<code>Prometheus</code>的配置文件，如下</p>

<pre><code class="language-yaml">apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-config
  namespace: kube-ops
data:
  prometheus.yml: |
    global:
      scrape_interval: 30s
      scrape_timeout: 30s
    scrape_configs:
    - job_name: 'prometheus'
      static_configs:
        - targets: ['localhost:9090']
    - job_name: 'kubernetes-nodes-cadvisor'
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      kubernetes_sd_configs:
      - api_servers:
        - 'https://10.43.0.1'
        in_cluster: true
        role: node
      relabel_configs:
      - action: labelmap
        regex: __meta_kubernetes_node_label_(.+)
      - source_labels: [__meta_kubernetes_role]
        action: replace
        target_label: kubernetes_role
      - source_labels: [__address__]
        regex: '(.*):10250'
        replacement: '${1}:4194'
        target_label: __address__
    - job_name: 'kubernetes-apiserver-cadvisor'
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      kubernetes_sd_configs:
      - api_servers:
        - 'https://10.43.0.1'
        in_cluster: true
        role: apiserver
      relabel_configs:
      - action: labelmap
        regex: __meta_kubernetes_node_label_(.+)
      - source_labels: [__meta_kubernetes_role]
        action: replace
        target_label: kubernetes_role
      - source_labels: [__address__]
        regex: '(.*):10250'
        replacement: '${1}:10255'
        target_label: __address__
    - job_name: 'kubernetes-node-exporter'
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      kubernetes_sd_configs:
      - api_servers:
        - 'https://10.43.0.1'
        in_cluster: true
        role: node
      relabel_configs:
      - action: labelmap
        regex: __meta_kubernetes_node_label_(.+)
      - source_labels: [__meta_kubernetes_role]
        action: replace
        target_label: kubernetes_role
      - source_labels: [__address__]
        regex: '(.*):10250'
        replacement: '${1}:31672'
        target_label: __address__
</code></pre>

<p>将以上配置文件保存为<code>prometheus-config.yaml</code>，然后执行命令：</p>

<pre><code class="language-shell">$ kubectl create -f prometheus-config.yaml
</code></pre>

<p>注意：</p>

<ul>
<li><code>job_name=kubernetes-apiserver-cadvisor</code>需要将10250端口替换成10255，10255端口是<code>kubelet</code>实现的<code>metrics</code>，可以在节点上面curl查看内容，<code>curl http://&lt;node_ip&gt;:10255/metrics</code></li>
<li><code>job_name=kubernetes-nodes-cadvisor</code>需要将10250端口替换成4194，4194同样是kubernetes集成的容器监控服务，在k8s 1.7版本之前的用10255端口即可，但是1.7版本后<code>cadvisor</code>监控的数据没有集成到<code>kubelet</code>的实现里面去了，<a href="https://github.com/kubernetes/kubernetes/issues/48483">这里一定要注意</a></li>
<li><code>job_name=kubernetes-node-exporter</code>中替换10250的端口是31672，该端口是<code>node-exporter</code>暴露的<code>NodePort</code>端口，这里需要根据实际情况填写。</li>
</ul>

<p>先部署<code>node-exporter</code>，为了能够收集每个节点的信息，所以我们这里使用<code>DaemonSet</code>的形式部署PODS：</p>

<pre><code class="language-yaml">---
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
  name: node-exporter
  namespace: kube-ops
  labels:
    k8s-app: node-exporter
spec:
  template:
    metadata:
      labels:
        k8s-app: node-exporter
    spec:
      containers:
      - image: prom/node-exporter
        name: node-exporter
        ports:
        - containerPort: 9100
          protocol: TCP
          name: http
---
apiVersion: v1
kind: Service
metadata:
  labels:
    k8s-app: node-exporter
  name: node-exporter
  namespace: kube-ops
spec:
  ports:
  - name: http
    port: 9100
    nodePort: 31672
    protocol: TCP
  type: NodePort
  selector:
    k8s-app: node-exporter
</code></pre>

<p>将以上文件保存为<code>node-exporter.yaml</code>，然后执行命令：</p>

<pre><code class="language-shell">$ kubectl create -f node-exporter.yaml
</code></pre>

<p>接下来通过<code>Deployment</code>部署<code>Prometheus</code>,yaml文件如下：</p>

<pre><code class="language-yaml">apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  labels:
    name: prometheus-deployment
  name: prometheus
  namespace: kube-ops
spec:
  replicas: 1
  template:
    metadata:
      labels:
        app: prometheus
    spec:
      containers:
      - image: prom/prometheus:v1.0.1
        name: prometheus
        command:
        - &quot;/bin/prometheus&quot;
        args:
        - &quot;-config.file=/etc/prometheus/prometheus.yml&quot;
        - &quot;-storage.local.path=/prometheus&quot;
        - &quot;-storage.local.retention=24h&quot;
        ports:
        - containerPort: 9090
          protocol: TCP
        volumeMounts:
        - mountPath: &quot;/prometheus&quot;
          name: data
          subPath: prometheus
        - mountPath: &quot;/etc/prometheus&quot;
          name: config-volume
        resources:
          requests:
            cpu: 100m
            memory: 100Mi
          limits:
            cpu: 200m
            memory: 1Gi
      volumes:
      - name: data
        emptyDir: {}
      - configMap:
          name: prometheus-config
        name: config-volume
</code></pre>

<p>将以上文件保存为<code>prometheus-deploy.yaml</code>，然后执行命令：</p>

<pre><code class="language-shell">$ kubectl create -f prometheus-deploy.yaml
</code></pre>

<p>接下来暴露服务以便可以访问<code>Prometheus</code>的UI界面，你可以通过<code>kubectl port-forward</code>将它暴露在本地：</p>

<pre><code class="language-shell">$ POD=`kubectl get pod -l app=prometheus -n kube-ops -o go-template --template '{{range .items}}{{.metadata.name}}{{end}}'`
$ kubectl port-forward $POD 9090:9090
</code></pre>

<p>然后用浏览器访问<code>http://localhost:9090</code>就可以访问到Prometheus的界面了。</p>

<p>我这里通过<code>ingress</code>暴露到外网，yaml文件如下：</p>

<pre><code class="language-yaml">apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: traefik-default-ingress
  annotations:
    kubernetes.io/ingress.class: &quot;traefik&quot;
spec:
  tls:
    - secretName: traefik-ssl
  rules:
  - host: prometheus.local  # 替换成你的域名
    http:
      paths:
      - path: /
        backend:
          serviceName: prometheus
          servicePort: 9090
</code></pre>

<p>将以上文件保存为<code>prometheus-ingress.yaml</code>，然后执行命令：</p>

<pre><code class="language-shell">$ kubectl create -f prometheus-ingress.yaml
</code></pre>

<p>然后就可以通过上面<code>traefik</code>中配置的域名进行访问了，可以切换到<code>Status</code>下面的<code>targets</code>查看我们采集的数据是否正常：
<img src="https://blog.qikqiak.com/img/posts/1508237027871.jpg" alt="Prometheus Targets" />
可以根据<code>targets</code>下面的提示信息对采集失败的数据进行修正。</p>

<h2 id="查询监控数据">查询监控数据</h2>

<p><code>Prometheus</code>提供了API的方式进行数据查询，同样可以使用query语言进行复杂的查询任务，在上面的WEB界面上提供了基本的查询和图形化的展示功能。</p>

<p>比如查询每个<code>POD</code>的CPU使用情况，查询条件如下：</p>

<pre><code class="language-shell">sum by (pod_name)( rate(container_cpu_usage_seconds_total{image!=&quot;&quot;, pod_name!=&quot;&quot;}[1m] ) )
</code></pre>

<p>注意其中的<code>pod_name</code>和<code>image</code>要根据自己采集的数据进行区分。</p>

<p><img src="https://blog.qikqiak.com/img/posts/1508237868056.jpg" alt="CPU使用" /></p>

<p>更多的查询条件可以参考<a href="https://prometheus.io/docs/introduction/overview/">Prometheus的文档</a>，将来也会逐步介绍，这里就不详细展开了。</p>

<p>这样通过在<code>Kubernetes</code>上部署<code>Prometheus</code>，在不修改集群的任何配置情况下实现了集群的基本监控功能。</p>

<h2 id="安装grafana">安装Grafana</h2>

<p><code>Prometheus</code>以及获取到了我们采集的数据，现在我们需要一个更加强大的图标展示工具，毫无疑问选择<code>grafana</code>，同样的，在<code>Kubernetes</code>环境下面进行安装，yaml文件如下:</p>

<pre><code class="language-yaml">apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: grafana
  namespace: kube-ops
spec:
  replicas: 1
  template:
    metadata:
      labels:
        k8s-app: grafana
        task: monitoring
    spec:
      containers:
      - name: grafana
        image: gcr.io/google_containers/heapster-grafana-amd64:v4.4.3
        ports:
        - containerPort: 3000
          protocol: TCP
        resources:
          limits:
            cpu: 200m
            memory: 256Mi
          requests:
            cpu: 100m
            memory: 100Mi
        volumeMounts:
        - name: ca-certificates
          mountPath: /etc/ssl/certs
          readOnly: true
        - name: grafana-data
          mountPath: /var
          subPath: grafana
        env:
        - name: INFLUXDB_HOST
          value: influxdb
        - name: INFLUXDB_SERVICE_URL
          value: http://influxdb.kube-ops.svc.cluster.local:8086
        - name: GF_SERVER_HTTP_PORT
          value: &quot;3000&quot;
        - name: GF_AUTH_BASIC_ENABLED
          value: &quot;false&quot;
        - name: GF_AUTH_ANONYMOUS_ENABLED
          value: &quot;true&quot;
        - name: GF_AUTH_ANONYMOUS_ORG_ROLE
          value: Admin
        - name: GF_SERVER_ROOT_URL
          # If you're only using the API Server proxy, set this value instead:
          # value: /api/v1/proxy/namespaces/kube-system/services/monitoring-grafana/
          value: /
      volumes:
      - name: ca-certificates
        hostPath:
          path: /etc/ssl/certs
      - name: grafana-data
        emptyDir: {}

---
apiVersion: v1
kind: Service
metadata:
  labels:
    kubernetes.io/cluster-service: 'true'
    kubernetes.io/name: grafana
  name: grafana
  namespace: kube-ops
spec:
  ports:
  - port: 3000
    targetPort: 3000
  selector:
    k8s-app: grafana
</code></pre>

<p>将以上文件保存为<code>grafana.yaml</code>，然后执行命令：</p>

<pre><code class="language-shell">$ kubectl create -f grafana.yaml
</code></pre>

<p>同样的你可以选择使用<code>kubectl port-forward</code>把端口暴露在本地，或者用<code>ingress</code>将服务暴露在外网进行访问。
访问<code>grafana</code>WEB界面，将我们上面的<code>Prometheus</code>添加到<code>grafana</code>数据源中去
<img src="https://blog.qikqiak.com/img/posts/1508238528462.jpg" alt="grafana数据源" /></p>

<p>然后添加我们的<code>Dashboard</code>，推荐使用<a href="https://grafana.com/dashboards/162">https://grafana.com/dashboards/162</a>，可以下载该页面的dashboard的json文件，然后直接导入到<code>grafana</code>中去，但是需要注意其中的一些参数，需要根据<code>prometheus</code>中采集到实际数据进行填写，比如我们这里采集到容器名是<code>name</code>，而不是<code>io_kubernetes_container_name</code>,最终展示界面如下：
<img src="https://blog.qikqiak.com/img/posts/1508238899864.jpg" alt="kubernetes monitor dashboard" /></p>

<h2 id="总结">总结</h2>

<p>未来将进一步完善<code>Prometheus</code>的使用：</p>

<ul>
<li>增加更多的监控数据源</li>
<li><a href="https://blog.qikqiak.com/post/alertmanager-of-prometheus-in-practice/">使用AlertManager实现异常提醒</a>（DONE）</li>
</ul>

<blockquote>
<p>上面用的<code>yaml</code>文件可以到<code>github</code>上查看<a href="https://github.com/cnych/k8s-repo/tree/master/prometheus">https://github.com/cnych/k8s-repo/tree/master/prometheus</a></p>
</blockquote>

<p>欢迎大家加入我们的知识星球：<code>Kubernetes</code>。
<img src="https://blog.qikqiak.com/img/xq.png" alt="知识星球" /></p>

<p>扫描下面的二维码(或微信搜索<code>k8s技术圈</code>)关注我们的微信公众帐号，在微信公众帐号中回复 <strong>加群</strong> 即可加入到我们的 kubernetes 讨论群里面共同学习。
<img src="https://blog.qikqiak.com/img/posts/qrcode_for_gh_d6dd87b6ceb4_430.jpg" alt="qrcode" /></p>
      </article>

      
        

<h3>相关文章</h3>
<ul style="margin-bottom: 25px;">
    
    <li><a href="https://blog.qikqiak.com/post/first-use-helm-on-kubernetes/">Kubernetes Helm 初体验</a></li>
    
    <li><a href="https://blog.qikqiak.com/post/understand-kubernetes-configmap-and-secrets/">kubernetes ConfigMap 和 Secrets</a></li>
    
    <li><a href="https://blog.qikqiak.com/post/add-authorization-for-kubernetes-dashboard/">为kubernetes dashboard访问用户添加权限控制</a></li>
    
</ul>

      

      
      <ul class="pager blog-pager">
        
          <li class="previous">
            <a href="https://blog.qikqiak.com/post/django-decorator-usage/" data-toggle="tooltip" data-placement="top" title="Python装饰器简介">&larr; Previous Post</a>
          </li>
        
        
          <li class="next">
            <a href="https://blog.qikqiak.com/post/kubernetes-rollout-update/" data-toggle="tooltip" data-placement="top" title="Kubernetes Deployment滚动升级">Next Post &rarr;</a>
          </li>
        
      </ul>
      

      

      
      <div id="git-comments"></div>
      <link rel="stylesheet" href="https://imsun.github.io/gitment/style/default.css">
      <script src="https://imsun.github.io/gitment/dist/gitment.browser.js"></script>
      <script>
      var gitment = new Gitment({
        id: 'kubernetes-monitor-prometheus-grafana',
        owner: 'cnych',
        repo: 'blog',
        oauth: {
          client_id: 'bdb76dbb2e9d0786e350',
          client_secret: 'b454b2a08013fd0e32013be7a63fa8fcb262b6c4',
        }
      })
      gitment.render('git-comments')
      </script>
      

    </div>
  </div>
</div>

    <footer>
  <div class="container">
    <div class="row">
      <div class="col-lg-8 col-lg-offset-2 col-md-10 col-md-offset-1">
        <ul class="list-inline text-center footer-links">
          
              <li>
                <a href="mailto:icnych@gmail.com" title="Email me">
                  <span class="fa-stack fa-lg">
                    <i class="fa fa-circle fa-stack-2x"></i>
                    <i class="fa fa-envelope fa-stack-1x fa-inverse"></i>
                  </span>
                </a>
              </li>
          
              <li>
                <a href="https://github.com/cnych" title="GitHub">
                  <span class="fa-stack fa-lg">
                    <i class="fa fa-circle fa-stack-2x"></i>
                    <i class="fa fa-github fa-stack-1x fa-inverse"></i>
                  </span>
                </a>
              </li>
          
              <li>
                <a href="https://weibo.com/cnych" title="微博">
                  <span class="fa-stack fa-lg">
                    <i class="fa fa-circle fa-stack-2x"></i>
                    <i class="fa fa-weibo fa-stack-1x fa-inverse"></i>
                  </span>
                </a>
              </li>
          
              <li>
                <a href="https://instagram.com/cnych" title="Instagram">
                  <span class="fa-stack fa-lg">
                    <i class="fa fa-circle fa-stack-2x"></i>
                    <i class="fa fa-instagram fa-stack-1x fa-inverse"></i>
                  </span>
                </a>
              </li>
          
          
          <li>
            <a href="https://blog.qikqiak.com/index.xml" title="RSS">
              <span class="fa-stack fa-lg">
                <i class="fa fa-circle fa-stack-2x"></i>
                <i class="fa fa-rss fa-stack-1x fa-inverse"></i>
              </span>
            </a>
          </li>
          
        </ul>
        <p class="credits copyright text-muted">
          

          &nbsp;&bull;&nbsp;
          2018

          
            &nbsp;&bull;&nbsp;
            <a href="https://blog.qikqiak.com/">阳明的博客</a>
          
        </p>
        
        <p class="credits theme-by text-muted">
          <a href="http://gohugo.io">Hugo v0.34</a> powered &nbsp;&bull;&nbsp; Theme by <a href="http://deanattali.com/beautiful-jekyll/">Beautiful Jekyll</a> adapted to <a href="https://github.com/halogenica/beautifulhugo">Beautiful Hugo</a>
          
        </p>
      </div>
    </div>
  </div>
</footer>

<script src="https://cdn.bootcss.com/KaTeX/0.7.1/katex.min.js"></script>
<script src="https://cdn.bootcss.com/KaTeX/0.7.1/contrib/auto-render.min.js"></script>
<script src="https://cdn.bootcss.com/jquery/1.12.4/jquery.min.js"></script>
<script src="https://cdn.bootcss.com/bootstrap/3.3.7/js/bootstrap.min.js"></script>
<script src="https://blog.qikqiak.com/js/main.min.js"></script>
<script src="https://blog.qikqiak.com/js/highlight.min.js"></script>
<script> hljs.initHighlightingOnLoad(); </script>
<script> renderMathInElement(document.body); </script>
<script src="https://blog.qikqiak.com/js/prism.js?t=123"></script>
<script src="https://cdn.bootcss.com/photoswipe/4.1.1/photoswipe.min.js"></script>
<script src="https://cdn.bootcss.com/photoswipe/4.1.1/photoswipe-ui-default.min.js"></script>
<script src="https://blog.qikqiak.com/js/load-photoswipe.min.js"></script>



<script async src="https://www.googletagmanager.com/gtag/js?id=UA-69668147-3"></script>
<script>
  window.dataLayer = window.dataLayer || [];
  function gtag(){dataLayer.push(arguments);}
  gtag('js', new Date());
  gtag('config', 'UA-69668147-3');
</script>

  </body>
</html>

